﻿using System.Threading;
using DM4;
using sscore;
using System;
using System.Collections.Generic;
using sscore.Utils;
using System.Text.RegularExpressions;
using System.IO;
using sscore.Requests;
using System.Net;

namespace AR
{
    public class AR : ModuleBase
    {
        private readonly string _offenderLink;
        private readonly string _imageLink;
        private const string StateName = "AR";
        private ResultFile _imagesResultFile;
        private static readonly string[] Types = { "Mappble", "NonMappble" };
        private string _ina_sec_csrf = "";

        List<string> _letters;
        const string LettersFileName = "Letters.txt";

        public AR()
        {
            //initialize main system constants
            ModuleCode = "SO_AR";
            Description = "AR Sex Offenders";
            Version = "1.0";

            //for creating sessions
            StartParam = 0;
            StopParam = Types.Length - 1;

            //initialize help 
            AddToHelp(DM4Consts.StartPosition, String.Format("{0}...{1}", StartParam, StopParam));
            AddToHelp(DM4Consts.EndPositon, String.Format("{0}...{1}", StartParam, StopParam));

            AddToHelp(DM4Consts.IsDownload, "true|false");
            AddToHelp(DM4Consts.IsDownloadImg, "true|false");
            AddToHelp(DM4Consts.InsertToDb, "false|true", "(Expected: servername, dbname)");
            AddToHelp(DM4Consts.ServerName, "");
            AddToHelp(DM4Consts.DbName, "");
            AddToHelp(DM4Consts.IsClearDb, "false|true");
            AddToHelp(DM4Consts.TestParam, "false|true");

            RootUrl = "http://www.acic.org/";
            //_searchUrl = RootUrl + "sor/SearchAction.cfm?PageNum_Subjects={0}&LNAME=&CITY=&FNAME=&COUNTY=&SORT=Name&ZIP=&PICS=";
            _offenderLink = String.Format("{0}offender-search/index.php?do:getById=1&id={{0}}", RootUrl);
            _imageLink = String.Format("{0}soff/images/{{0}}.jpg", RootUrl);
        }

        private void SetParametersDefaults()
        {
            Params.CreateBooleanParam(DM4Consts.IsDownload, true);
            Params.CreateBooleanParam(DM4Consts.IsDownloadImg, true);
            Params.CreateBooleanParam(DM4Consts.InsertToDb, false);
            Params.CreateBooleanParam(DM4Consts.IsClearDb, false);
            Params.CreateStringParam(DM4Consts.DbName, "");
            Params.CreateStringParam(DM4Consts.ServerName, "");
            Params.CreateIntegerParam(DM4Consts.StartPosition, StartParam);
            Params.CreateIntegerParam(DM4Consts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
            Params.CreateIntegerParam(DM4Consts.EndPositon, StopParam);
            Params.CreateIntegerParam(DmConsts.LastChars, 0);
            Params.CreateIntegerParam(DmConsts.LastRecord, 0);
            Params.CreateIntegerParam(DmConsts.LastPage, 0);
        }

        public override OperationResult Init(IDM4Core iCore, DmTaskParams param)
        {
            //use basic initialization sequence
            base.Init(iCore, param);

            SetParametersDefaults();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult Start()
        {
            DoWork();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult DoWork()
        {
            OperationResult or = new OperationResult { Result = DM4Consts.ErrorUnknown };

            if (!Ct.IsCancellationRequested)
            {
                WriteLog("Start parameters:", MessageType.Warning);
                foreach (KeyValuePair<string, string> kvp in Params.GetDictionary()) //get all existing parameters
                    WriteLog(String.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value), MessageType.Warning);

                if (Params.GetBooleanParam(DM4Consts.IsDownload))
                    if (CreateLettersList())
                        Load();

                if (Params.GetBooleanParam(DM4Consts.InsertToDb))
                    if ((Params.GetStringParam(DM4Consts.DbName) == "") || (Params.GetStringParam(DM4Consts.ServerName) == ""))
                    {
                        or.Message = String.Format("{0} or {1} is empty", DM4Consts.DbName, DM4Consts.ServerName);
                        ResultCode = DM4Consts.SessionResultCodeError;
                        ResultMessage = or.Message;
                        UpdateStatus(DmConsts.Error);

                        WriteErrorLog(or.Message); //writting to error and session log   
                        return or;
                    }
                    else
                    {
                        //insert offenders to db 
                        UpdateStatus(DM4Consts.SessionInsertToDb);

                        Offenders.OffInsertToDatabase(StateName, Params.GetStringParam(DmConsts.DbName),
                             Params.GetStringParam(DmConsts.ServerName), ResultDir, Params.GetBooleanParam(DmConsts.IsClearDb),
                             WriteLog);

                    }

                //check if need abort task
                if (Ct.IsCancellationRequested)
                {
                    ResultCode = DM4Consts.SessionResultTerminateByUser;
                    ResultMessage = "Session terminated by user";
                    UpdateStatus(DM4Consts.SessionWait);
                    WriteLog("Session aborted by user", MessageType.Cool);
                }
                else
                {
                    UpdateProgress(1);
                    ResultCode = DM4Consts.SessionResultCodeOk;
                    ResultMessage = String.Format("Session complete, errors : {0}, records : {1}", TaskParams.ErrorCount, TaskParams.RecordCount);
                    UpdateStatus(DM4Consts.SessionComplete);
                    WriteLog("------------Finish----------------", MessageType.Cool);
                }
            }            
            else
            {
                ResultCode = DM4Consts.SessionResultTerminateByUser;
                ResultMessage = "Session terminated by user";
                UpdateStatus(DM4Consts.SessionWait);
                WriteLog("Session aborted by user", MessageType.Cool);
            }

            return or;
        }

        private void Load()
        {
            string resultFileName = String.Format("{0}\\{1}.{2}", ResultDir, TaskParams.SessionId, DmConsts.ResultFilesExtension);
            bool workFlag = true;

            ResultFile = new ResultFile(resultFileName, OffendersResult.GetHeader());
            _imagesResultFile = new ResultFile(String.Format("{0}\\{1}_images.{2}", ResultDir, TaskParams.SessionId, DM4Consts.ResultFilesExtension), "FileName|FileUrl");
            try
            {
                while (workFlag)
                {
                    //рабочий цикл
                    string error;

                    try
                    {
                        error = "";  //Required!!  /*if (!String.IsNullOrEmpty(error)) --> restart thread; else --> download finished*/

                        WriteLog("Start download...");
                        Searching();
                    }
                    catch (Exception e)
                    {
                        error = e.Message;
                    }

                    if (!String.IsNullOrEmpty(error))
                    {
                        Thread.Sleep(30000);
                        WriteErrorLog(error);
                        WriteLog("Try continue work...", MessageType.Warning);
                    }
                    else
                    {
                        workFlag = false;  //все нормально откачало, заканчиваем работу
                    }
                }
            }
            finally
            {
                ResultFile.Close();
                _imagesResultFile.Close();
            }
        }

        private string GetCookies()
        {
            WriteLog("get cookies");
            string html = GetRequest.ReturnCookies(String.Format("{0}offender-search/index.php", RootUrl), Rpm);
            Rpm.Cookies = RegExpr.Parse(Rpm.Cookies, @"(?si:^([^;]*?);)");
            _ina_sec_csrf = RegExpr.Parse(html, @"(?si:name='ina_sec_csrf' value=""([^""]*?)"")");

            return html;
        }

        private void Searching()
        {
            if (Test)
            {
                Params.SetIntegerParam(DmConsts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
                Params.SetIntegerParam(DmConsts.EndPositon, Params.GetIntegerParam(DmConsts.StartPosition));
            }
            if (Params.GetIntegerParam(DmConsts.EndPositon) > (Types.Length - 1))
                Params.SetIntegerParam(DmConsts.EndPositon, Types.Length - 1);

            GetCookies();
            for (int indSearchParam = Params.GetIntegerParam(DmConsts.LastPosition); indSearchParam <= Params.GetIntegerParam(DmConsts.EndPositon); indSearchParam++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastPosition, indSearchParam);

                //check param and update progress
                if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                    throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                        Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                        DmConsts.StartPosition, DmConsts.EndPositon));
                if (Params.GetIntegerParam(DmConsts.LastPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                    throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                        Params.GetIntegerParam(DmConsts.LastPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                        DmConsts.StartPosition, DmConsts.EndPositon));
                if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.LastPosition))
                    throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                        Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.LastPosition),
                        DmConsts.StartPosition, DmConsts.LastPosition));
                UpdateProgress(((double)(Params.GetIntegerParam(DmConsts.LastPosition) - Params.GetIntegerParam(DmConsts.StartPosition))) / (Params.GetIntegerParam(DmConsts.EndPositon) - Params.GetIntegerParam(DmConsts.StartPosition) + 1));

                WriteLog(String.Format("start process type [{0}][{1}]", Params.GetIntegerParam(DmConsts.LastPosition), Types[Params.GetIntegerParam(DmConsts.LastPosition)]));

                if (Params.GetIntegerParam(DmConsts.LastPosition) == 0)
                    LoadMappbleOffenders();
                else
                    LoadNonMappbleOffenders();
            }
        }

        private void LoadNonMappbleOffenders()
        {
            string html = GetRedirectRequest.WithCookies(String.Format(
                "{0}offender-search/index.php?ina_sec_csrf={1}&do:getNonMappableData=1&sEcho=3&iColumns=11&sColumns=&iDisplayStart={2}&iDisplayLength=100&sSearch=&bEscapeRegex=true&sSearch_0=&bEscapeRegex_0=true&bSearchable_0=true&sSearch_1=&bEscapeRegex_1=true&bSearchable_1=true&sSearch_2=&bEscapeRegex_2=true&bSearchable_2=true&sSearch_3=&bEscapeRegex_3=true&bSearchable_3=true&sSearch_4=&bEscapeRegex_4=true&bSearchable_4=true&sSearch_5=&bEscapeRegex_5=true&bSearchable_5=true&sSearch_6=&bEscapeRegex_6=true&bSearchable_6=true&sSearch_7=&bEscapeRegex_7=true&bSearchable_7=true&sSearch_8=&bEscapeRegex_8=true&bSearchable_8=true&sSearch_9=&bEscapeRegex_9=true&bSearchable_9=true&sSearch_10=&bEscapeRegex_10=true&bSearchable_10=true"
                , RootUrl, _ina_sec_csrf, Params.GetIntegerParam(DmConsts.LastRecord)), Rpm);

            int totalCount = Convert.ToInt32(RegExpr.Parse(html, @"(?si:""iTotalRecords""\s*:\s*([\d]*?)\s*,)"));
            do
            {
                if (Ct.IsCancellationRequested)
                    break;

                if (Params.GetIntegerParam(DmConsts.StartPosition) == 0)
                    UpdateProgress((double)Params.GetIntegerParam(DmConsts.LastRecord) / totalCount * (double)0.5 + (double)0.5);
                if (Params.GetIntegerParam(DmConsts.StartPosition) == 1)
                    UpdateProgress((double)Params.GetIntegerParam(DmConsts.LastRecord) / totalCount);

                html = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(html.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
                ParseNonMappbaleOffemders(html);

                if (totalCount > Params.GetIntegerParam(DmConsts.LastRecord))
                {
                    html = GetRedirectRequest.WithCookies(String.Format(
                           "{0}offender-search/index.php?ina_sec_csrf={1}&do:getNonMappableData=1&sEcho=3&iColumns=11&sColumns=&iDisplayStart={2}&iDisplayLength=100&sSearch=&bEscapeRegex=true&sSearch_0=&bEscapeRegex_0=true&bSearchable_0=true&sSearch_1=&bEscapeRegex_1=true&bSearchable_1=true&sSearch_2=&bEscapeRegex_2=true&bSearchable_2=true&sSearch_3=&bEscapeRegex_3=true&bSearchable_3=true&sSearch_4=&bEscapeRegex_4=true&bSearchable_4=true&sSearch_5=&bEscapeRegex_5=true&bSearchable_5=true&sSearch_6=&bEscapeRegex_6=true&bSearchable_6=true&sSearch_7=&bEscapeRegex_7=true&bSearchable_7=true&sSearch_8=&bEscapeRegex_8=true&bSearchable_8=true&sSearch_9=&bEscapeRegex_9=true&bSearchable_9=true&sSearch_10=&bEscapeRegex_10=true&bSearchable_10=true"
                           , RootUrl, _ina_sec_csrf, Params.GetIntegerParam(DmConsts.LastRecord)), Rpm);
                }
                else break;

            } while (true);

            Params.SetIntegerParam(DmConsts.LastRecord, 0);
        }

        private void ParseNonMappbaleOffemders(string html)
        {
            MatchCollection mc = RegExpr.ParseMatchCollection(html, @"(?si:images/([^<>]*?).jpg\\"">"",""(.*?)"",""(.*?)"",""(.*?)"",""(.*?)"",""(.*?)"",""(.*?)"",""(.*?)"",""(.*?)"",""(.*?)"",""(.*?)""\s*\x5D)");
            WriteLog(String.Format("found [{0}] matches", mc.Count));

            foreach (Match matchLoc in mc)
            {
                using (OffendersResult or = new OffendersResult())
                {
                    or.Source_State = StateName;

                    string htmlImageId = matchLoc.Groups[1].Value.Trim();
                    or.fullname = matchLoc.Groups[2].Value.Trim();
                    or.Address = matchLoc.Groups[3].Value.Trim();
                    or.Birthdate = matchLoc.Groups[4].Value.Trim();
                    or.sex = matchLoc.Groups[5].Value.Trim();
                    or.Ethnicity = matchLoc.Groups[6].Value.Trim();
                    or.Crime = matchLoc.Groups[8].Value.Trim();
                    or.eye_color = matchLoc.Groups[9].Value.Trim();
                    or.hair_color = matchLoc.Groups[10].Value.Trim();
                    or.marks = matchLoc.Groups[11].Value.Trim();

                    if (!string.IsNullOrEmpty(htmlImageId))
                    {
                        or.Image_ID = LocalDownloadImage(htmlImageId);
                        if (!string.IsNullOrEmpty(or.Image_ID))
                            or.ImageLink = String.Format(_imageLink, htmlImageId);
                    }
                    ResultFile.Write(or.GetValues());
                }
                UpdateRecordCount(1);
                Params.SetIntegerParam(DmConsts.LastRecord, Params.GetIntegerParam(DmConsts.LastRecord) + 1);
            }

        }

        private string LocalDownloadImage(string htmlImageId)
        {
            string result = Guid.NewGuid() + ".jpg";
            string url = String.Format(_imageLink, htmlImageId);
            try
            {
                Offenders.DownloadImageWithExceptions(_imagesResultFile
                    , String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, result)
                    , url, TaskParams.SessionId);
            }
            catch (WebException webEx)
            {
                if (((HttpWebResponse)webEx.Response).StatusCode == HttpStatusCode.NotFound
                    || ((HttpWebResponse)webEx.Response).StatusCode == HttpStatusCode.InternalServerError)
                    result = "";
                else throw new Exception("Error occured while downloading image: " + url, webEx);
            }
            catch (Exception ex)
            {
                //если произошла загрузка битой картинки
                if (ex.Message == "Parameter is not valid." || ex.StackTrace.StartsWith("   at System.Drawing.Bitmap..ctor(String filename)"))
                    result = "";
                else
                    throw new Exception("Error occured while downloading image: " + url, ex);
            }

            return result;
        }

        private bool CreateLettersList()
        {
            _letters = new List<string>();

            FileInfo fileInfo = new FileInfo(String.Format("{0}/{1}", SettingsDir, LettersFileName));
            if (fileInfo.Exists)
            {
                try
                {
                    using (StreamReader fileReader = fileInfo.OpenText())
                    {
                        String line;
                        while ((line = fileReader.ReadLine()) != null)
                            _letters.Add(line.Trim());
                        fileReader.Close();
                    }
                    return true;
                }
                catch { return false; }
            }
            return false;
        }

        private void LoadMappbleOffenders()
        {
            for (int curLetterIndex = Params.GetIntegerParam(DmConsts.LastChars); curLetterIndex < _letters.Count; curLetterIndex++)
            {
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastChars, curLetterIndex);

                if (Params.GetIntegerParam(DmConsts.StartPosition) == 1)
                    UpdateProgress((double)Params.GetIntegerParam(DmConsts.LastChars) / _letters.Count * (double)0.5);

                if (Params.GetIntegerParam(DmConsts.StartPosition) == 0)
                    UpdateProgress((double)Params.GetIntegerParam(DmConsts.LastChars) / _letters.Count);

                WriteLog(String.Format("navigate to letters [{0}][{1}]"
                        , Params.GetIntegerParam(DmConsts.LastChars), _letters[Params.GetIntegerParam(DmConsts.LastChars)]));


                string html = GetRedirectRequest.WithCookies(String.Format(
                    "{0}offender-search/index.php?ina_sec_csrf={1}&do:searchList=1&lname={2}&fname=&page={3}"
                    , RootUrl, _ina_sec_csrf, _letters[Params.GetIntegerParam(DmConsts.LastChars)], Params.GetIntegerParam(DmConsts.LastPage)), Rpm);

                if (html == "NULL")
                    WriteLog("no records found");
                else
                {
                    NavigateByMappbleOffendersResultPages(html);
                    if (Test) break;
                }
            }//for (int curLetterIndex = FLastPositionParam; curLetterIndex <= FEndPositionParam; curLetterIndex++)
            Params.SetIntegerParam(DmConsts.LastChars, 0);
        }

        private void NavigateByMappbleOffendersResultPages(string html)
        {            
            int totalCount = Convert.ToInt32(RegExpr.Parse(html, @"(?si:;totalCount\s*=\s*([\d]*?);)"));
            WriteLog(String.Format("found [{0}] record(s)", totalCount));

            do
            {
                if (Ct.IsCancellationRequested)
                    break;

                html = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(html.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
                ParseMappbleOffendersResultPage(html);

                if (totalCount > (Params.GetIntegerParam(DmConsts.LastPage) + 1) * 20)
                {
                    Params.SetIntegerParam(DmConsts.LastPage, Params.GetIntegerParam(DmConsts.LastPage) + 1);
                    html = GetRedirectRequest.WithCookies(String.Format("{0}offender-search/index.php?ina_sec_csrf={1}&do:searchList=1&lname={2}&fname=&page={3}"
                        , RootUrl, _ina_sec_csrf, _letters[Params.GetIntegerParam(DmConsts.LastChars)], Params.GetIntegerParam(DmConsts.LastPage)), Rpm);                    
                }
                else break;

            } while (true);
            Params.SetIntegerParam(DmConsts.LastPage, 0);
        }

        private void ParseMappbleOffendersResultPage(string html)
        {
            string offenderHtml;
            int iOffender = 0;

            MatchCollection mc = RegExpr.ParseMatchCollection(html, @"(?si:\s*""id""\s*:\s*\x5B([\d]+)\x5D)");
            WriteLog(String.Format("found [{0}] matches", mc.Count));
            if (mc.Count == 0)
                throw new Exception(String.Format("Result page: {0}{1}{2}", DmConsts.ExcNeedRestart, Environment.NewLine, html));

            foreach (Match matchLoc in mc)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                if (iOffender >= Params.GetIntegerParam(DmConsts.LastRecord))
                {
                    string lookupKey = matchLoc.Groups[1].Value;

                    offenderHtml = GetRedirectRequest.WithCookies(String.Format(_offenderLink, lookupKey), Rpm);
                    if (!RegExpr.IsMatch(offenderHtml, @"(?si:>\s*Physical Description\s*<)"))
                        throw new Exception("Mappble offenders page: " + DmConsts.ExcNeedRestart);

                    WriteLog(String.Format("ParseOffender {0} ", lookupKey));
                    ParseMappbleOffender(offenderHtml, lookupKey);

                    UpdateRecordCount(1);
                    Params.SetIntegerParam(DmConsts.LastRecord, Params.GetIntegerParam(DmConsts.LastRecord) + 1);
                }
                iOffender++;
            }

            Params.SetIntegerParam(DmConsts.LastRecord, 0);

        }

        private void ParseMappbleOffender(string offenderHtml, string lookupKey)
        {
            offenderHtml = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(offenderHtml.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
            using (OffendersResult or = new OffendersResult())
            {
                or.Source_State = StateName;
                or.Source_id = lookupKey;
                or.offenderLink = String.Format(_offenderLink, lookupKey);

                or.fullname = RegExpr.Parse(offenderHtml, @"(?si:>\s*Name\s*:\s*</strong>\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.Address = RegExpr.Parse(offenderHtml, @"(?si:>\s*Address\s*</th>\s*</tr>\s*</thead>\s*<tr[^<>]*?>\s*<td>(.*?)<)").Trim();
                or.Birthdate = RegExpr.Parse(offenderHtml, @"(?si:\x28DOB\s*:(.*?)\x29)").Trim();
                or.Ethnicity = RegExpr.Parse(offenderHtml, @"(?si:>\s*Race\s*:\s*</strong>\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.eye_color = RegExpr.Parse(offenderHtml, @"(?si:>\s*Eyes\s*:\s*</strong>\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.hair_color = RegExpr.Parse(offenderHtml, @"(?si:>\s*Hair\s*:\s*</strong>\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.sex = RegExpr.Parse(offenderHtml, @"(?si:>\s*Sex\s*:\s*</strong>\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.marks = RegExpr.Parse(offenderHtml, @"(?si:>\s*Scars/Tattoos\s*:\s*</strong>\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.Risk = RegExpr.Parse(offenderHtml, @"(?si:>\s*Level\s*:\s*</strong>\s*</td>\s*<td>(.*?)</td>)");

                MatchCollection mc = RegExpr.ParseMatchCollection(
                    RegExpr.Parse(offenderHtml, @"(?si:Offenses\s*</th>(.*?)</table>)")
                    , @"(?si:<td>(.*?)</td>)");

                for (int i = 0; i < mc.Count; i++)
                    or.Crime += mc[i].Groups[1].Value.Trim() + ";";

                string htmlImageId = RegExpr.Parse(offenderHtml, "(?si:/images/(.*?).jpg)");
                or.Image_ID = LocalDownloadImage(htmlImageId);
                if (or.Image_ID != "")
                    or.ImageLink = String.Format(_imageLink, htmlImageId);

                ResultFile.Write(or.GetValues());
            }
        }
    }
}
